{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "99512c9b-e4c1-4249-9598-9b2eb77f2b05",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-01T11:18:57.381810Z",
     "start_time": "2025-07-01T11:18:55.587648Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "【开始】加载原始数据...\n",
      " 开始时间：2025-07-01 19:48:17.357040\n",
      " 原始数据加载完成：共 1581278 行，26 列\n",
      " 已采样前 100000 条数据\n",
      "\n",
      " 数据质量报告：\n",
      "                         Missing Count  Missing Rate (%)  Unique Count  \\\n",
      "MktCoupons                           0               0.0             3   \n",
      "OriginCityMarketID                   0               0.0            38   \n",
      "DestCityMarketID                     0               0.0            43   \n",
      "OriginAirportID                      0               0.0            56   \n",
      "DestAirportID                        0               0.0            61   \n",
      "Carrier                              0               0.0            23   \n",
      "NonStopMiles                         0               0.0           244   \n",
      "RoundTrip                            0               0.0             2   \n",
      "ODPairID                             0               0.0            87   \n",
      "Pax                                  0               0.0            87   \n",
      "CarrierPax                           0               0.0           264   \n",
      "Average_Fare                         0               0.0           413   \n",
      "Market_share                         0               0.0           261   \n",
      "Market_HHI                           0               0.0            84   \n",
      "LCC_Comp                             0               0.0             2   \n",
      "Multi_Airport                        0               0.0             2   \n",
      "Circuity                             0               0.0          1638   \n",
      "Slot                                 0               0.0             2   \n",
      "Non_Stop                             0               0.0             2   \n",
      "MktMilesFlown                        0               0.0            87   \n",
      "OriginCityMarketID_freq              0               0.0            38   \n",
      "DestCityMarketID_freq                0               0.0            43   \n",
      "OriginAirportID_freq                 0               0.0            56   \n",
      "DestAirportID_freq                   0               0.0            61   \n",
      "Carrier_freq                         0               0.0            23   \n",
      "ODPairID_freq                        0               0.0            87   \n",
      "\n",
      "                        Data Type  \n",
      "MktCoupons                  int64  \n",
      "OriginCityMarketID          int64  \n",
      "DestCityMarketID            int64  \n",
      "OriginAirportID             int64  \n",
      "DestAirportID               int64  \n",
      "Carrier                     int64  \n",
      "NonStopMiles              float64  \n",
      "RoundTrip                 float64  \n",
      "ODPairID                    int64  \n",
      "Pax                       float64  \n",
      "CarrierPax                float64  \n",
      "Average_Fare              float64  \n",
      "Market_share              float64  \n",
      "Market_HHI                float64  \n",
      "LCC_Comp                    int64  \n",
      "Multi_Airport               int64  \n",
      "Circuity                  float64  \n",
      "Slot                        int64  \n",
      "Non_Stop                  float64  \n",
      "MktMilesFlown             float64  \n",
      "OriginCityMarketID_freq   float64  \n",
      "DestCityMarketID_freq     float64  \n",
      "OriginAirportID_freq      float64  \n",
      "DestAirportID_freq        float64  \n",
      "Carrier_freq              float64  \n",
      "ODPairID_freq             float64  \n",
      "\n",
      " 元信息记录：\n",
      " - source_file: data/processed/MarketFarePredictionData.csv\n",
      " - sample_size: 100000\n",
      " - columns_count: 26\n",
      " - loaded_time: 2025-07-01 19:48:17\n",
      " - output_file: data/raw/raw_data.parquet\n",
      " 数据已保存至：data/raw/raw_data.parquet\n",
      "【完成】数据加载流程结束\n",
      "\n",
      " 数据预览：\n",
      "   MktCoupons  OriginCityMarketID  DestCityMarketID  OriginAirportID  \\\n",
      "0           2                 178               152              170   \n",
      "1           2                 178               152              170   \n",
      "2           2                 178               152              170   \n",
      "3           2                 178               152              170   \n",
      "4           2                 178               152              170   \n",
      "\n",
      "   DestAirportID  Carrier  NonStopMiles  RoundTrip  ODPairID    Pax  ...  \\\n",
      "0            255        6        1807.0        1.0      4035  136.0  ...   \n",
      "1            194       20        1798.0        1.0      4035  136.0  ...   \n",
      "2            260        6        1784.0        0.0      4035  136.0  ...   \n",
      "3            255        6        1807.0        1.0      4035  136.0  ...   \n",
      "4            194       20        1798.0        1.0      4035  136.0  ...   \n",
      "\n",
      "   Circuity  Slot  Non_Stop  MktMilesFlown  OriginCityMarketID_freq  \\\n",
      "0  1.367460     0       0.0    1992.449761                 0.004138   \n",
      "1  1.051724     0       0.0    1992.449761                 0.004138   \n",
      "2  1.034753     0       0.0    1992.449761                 0.004138   \n",
      "3  1.029884     0       0.0    1992.449761                 0.004138   \n",
      "4  1.062291     0       0.0    1992.449761                 0.004138   \n",
      "\n",
      "   DestCityMarketID_freq  OriginAirportID_freq  DestAirportID_freq  \\\n",
      "0               0.039783              0.004138            0.022049   \n",
      "1               0.039783              0.004138            0.008368   \n",
      "2               0.039783              0.004138            0.009366   \n",
      "3               0.039783              0.004138            0.022049   \n",
      "4               0.039783              0.004138            0.008368   \n",
      "\n",
      "   Carrier_freq  ODPairID_freq  \n",
      "0      0.116826       0.000132  \n",
      "1      0.307651       0.000132  \n",
      "2      0.116826       0.000132  \n",
      "3      0.116826       0.000132  \n",
      "4      0.307651       0.000132  \n",
      "\n",
      "[5 rows x 26 columns]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "def load_data(file_path, sample_size=None, output_path=None):\n",
    "    print(\"【开始】加载原始数据...\")\n",
    "\n",
    "    start_time = datetime.now()\n",
    "    print(f\" 开始时间：{start_time}\")\n",
    "\n",
    "    df = pd.read_csv(file_path)\n",
    "    original_shape = df.shape\n",
    "    print(f\" 原始数据加载完成：共 {original_shape[0]} 行，{original_shape[1]} 列\")\n",
    "\n",
    "    if sample_size:\n",
    "        df = df.head(sample_size).copy()\n",
    "        print(f\" 已采样前 {sample_size} 条数据\")\n",
    "\n",
    "    missing_summary = df.isnull().sum()\n",
    "    missing_rate = (missing_summary / df.shape[0]) * 100\n",
    "    quality_report = pd.DataFrame({\n",
    "        'Missing Count': missing_summary,\n",
    "        'Missing Rate (%)': missing_rate,\n",
    "        'Unique Count': df.nunique(),\n",
    "        'Data Type': df.dtypes\n",
    "    })\n",
    "    print(\"\\n 数据质量报告：\")\n",
    "    print(quality_report)\n",
    "\n",
    "    metadata = {\n",
    "        \"source_file\": file_path,\n",
    "        \"sample_size\": sample_size if sample_size else original_shape[0],\n",
    "        \"columns_count\": original_shape[1],\n",
    "        \"loaded_time\": start_time.strftime(\"%Y-%m-%d %H:%M:%S\"),\n",
    "        \"output_file\": output_path if output_path else \"Not saved\"\n",
    "    }\n",
    "    print(\"\\n 元信息记录：\")\n",
    "    for key, value in metadata.items():\n",
    "        print(f\" - {key}: {value}\")\n",
    "\n",
    "    if output_path:\n",
    "        os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
    "        df.to_parquet(output_path)\n",
    "        print(f\" 数据已保存至：{output_path}\")\n",
    "\n",
    "    print(\"【完成】数据加载流程结束\")\n",
    "    return df\n",
    "\n",
    "# 示例调用\n",
    "input_path = \"data/processed/MarketFarePredictionData.csv\"\n",
    "output_path = \"data/raw/raw_data.parquet\"\n",
    "raw_data = load_data(input_path, sample_size=100_000, output_path=output_path)\n",
    "print(\"\\n 数据预览：\")\n",
    "print(raw_data.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7c96a51a-9864-4051-9820-b39bf460941d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-01T11:37:12.853169Z",
     "start_time": "2025-07-01T11:37:10.321589Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🔄 正在进行数据预处理...\n",
      "【开始】数据预处理...\n",
      "✂️ 检测到 96258 条重复记录，正在删除...\n",
      "✅ 分类变量： ['Carrier', 'RoundTrip', 'ODPairID', 'LCC_Comp', 'Multi_Airport', 'Slot', 'Non_Stop', 'OriginCityMarketID', 'DestCityMarketID', 'OriginAirportID', 'DestAirportID']\n",
      "✅ 数值变量： ['MktCoupons', 'NonStopMiles', 'Pax', 'CarrierPax', 'Average_Fare', 'Market_share', 'Market_HHI', 'Circuity', 'MktMilesFlown', 'OriginCityMarketID_freq', 'DestCityMarketID_freq', 'OriginAirportID_freq', 'DestAirportID_freq', 'Carrier_freq', 'ODPairID_freq']\n",
      "📊 processed_array shape: (3742, 333)\n",
      "\n",
      "🔍 开始异常值检测...\n",
      "🔍 MktCoupons 发现 1102 个异常值（阈值：2.00 ~ 2.00）\n",
      "🔍 NonStopMiles 发现 9 个异常值（阈值：-903.50 ~ 3740.50）\n",
      "🔍 Pax 发现 0 个异常值（阈值：-663353.00 ~ 1333023.00）\n",
      "🔍 CarrierPax 发现 120 个异常值（阈值：-129967.50 ~ 282004.50）\n",
      "🔍 Average_Fare 发现 103 个异常值（阈值：43.28 ~ 384.69）\n",
      "🔍 Market_share 发现 0 个异常值（阈值：-0.32 ~ 1.04）\n",
      "🔍 Market_HHI 发现 155 个异常值（阈值：-763.60 ~ 8199.76）\n",
      "🔍 Circuity 发现 292 个异常值（阈值：0.58 ~ 1.71）\n",
      "🔍 MktMilesFlown 发现 9 个异常值（阈值：-940.64 ~ 3883.15）\n",
      "🔍 OriginCityMarketID_freq 发现 0 个异常值（阈值：-0.00 ~ 0.07）\n",
      "🔍 DestCityMarketID_freq 发现 0 个异常值（阈值：-0.03 ~ 0.11）\n",
      "🔍 OriginAirportID_freq 发现 0 个异常值（阈值：-0.02 ~ 0.07）\n",
      "🔍 DestAirportID_freq 发现 0 个异常值（阈值：-0.01 ~ 0.04）\n",
      "🔍 Carrier_freq 发现 0 个异常值（阈值：-0.21 ~ 0.62）\n",
      "🔍 ODPairID_freq 发现 345 个异常值（阈值：-0.00 ~ 0.00）\n",
      "\n",
      "📊 预处理后数据质量报告：\n",
      "              Missing Count  Missing Rate (%)  Unique Count Data Type\n",
      "MktCoupons                0               0.0             3   float64\n",
      "NonStopMiles              0               0.0           244   float64\n",
      "Pax                       0               0.0            87   float64\n",
      "CarrierPax                0               0.0           264   float64\n",
      "Average_Fare              0               0.0           413   float64\n",
      "...                     ...               ...           ...       ...\n",
      "x10_72                    0               0.0             2   float64\n",
      "x10_74                    0               0.0             2   float64\n",
      "x10_81                    0               0.0             2   float64\n",
      "x10_90                    0               0.0             2   float64\n",
      "x10_98                    0               0.0             2   float64\n",
      "\n",
      "[333 rows x 4 columns]\n",
      "【完成】数据预处理完成\n",
      "✅ 已保存预处理后的数据到 data/preprocessed/cleaned_data.parquet\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "def detect_outliers(df, numeric_features, method='iqr'):\n",
    "    \"\"\"\n",
    "    使用箱线图方法识别数值型字段的异常值\n",
    "    :param df: 输入DataFrame\n",
    "    :param numeric_features: 数值列列表\n",
    "    :param method: 检测方法，默认 'iqr'\n",
    "    :return: 包含异常统计的字典\n",
    "    \"\"\"\n",
    "    outlier_report = {}\n",
    "\n",
    "    # 创建输出目录\n",
    "    os.makedirs(\"reports\", exist_ok=True)\n",
    "\n",
    "    for col in numeric_features:\n",
    "        q1 = df[col].quantile(0.25)\n",
    "        q3 = df[col].quantile(0.75)\n",
    "        iqr = q3 - q1\n",
    "        lower_bound = q1 - 1.5 * iqr\n",
    "        upper_bound = q3 + 1.5 * iqr\n",
    "\n",
    "        outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]\n",
    "        outlier_count = len(outliers)\n",
    "        outlier_report[col] = {\n",
    "            \"outlier_count\": outlier_count,\n",
    "            \"lower_bound\": lower_bound,\n",
    "            \"upper_bound\": upper_bound\n",
    "        }\n",
    "\n",
    "        print(f\" {col} 发现 {outlier_count} 个异常值（阈值：{lower_bound:.2f} ~ {upper_bound:.2f}）\")\n",
    "\n",
    "        # 绘制箱线图并保存\n",
    "        plt.figure(figsize=(6, 2))\n",
    "        sns.boxplot(x=df[col])\n",
    "        plt.title(f\"{col} Boxplot\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"reports/outliers_{col}.png\")\n",
    "        plt.close()\n",
    "\n",
    "    return outlier_report\n",
    "\n",
    "\n",
    "def preprocess_data(df):\n",
    "    \"\"\"\n",
    "    数据预处理流程：\n",
    "        - 缺失值填充\n",
    "        - 标准化数值型字段\n",
    "        - One-Hot编码分类变量\n",
    "        - 去重处理\n",
    "        - 异常值检测\n",
    "    :param df: 原始DataFrame\n",
    "    :return: 预处理后的DataFrame\n",
    "    \"\"\"\n",
    "    print(\"【开始】数据预处理...\")\n",
    "\n",
    "    # 显式复制一份，避免后续警告\n",
    "    df = df.copy()\n",
    "\n",
    "    # 去重处理\n",
    "    if df.duplicated().sum() > 0:\n",
    "        print(f\" 检测到 {df.duplicated().sum()} 条重复记录，正在删除...\")\n",
    "        df = df.drop_duplicates()\n",
    "    else:\n",
    "        print(\" 未发现重复记录\")\n",
    "\n",
    "    # 手动定义分类列\n",
    "    categorical_features = [\n",
    "        'Carrier', 'RoundTrip', 'ODPairID', 'LCC_Comp', 'Multi_Airport',\n",
    "        'Slot', 'Non_Stop', 'OriginCityMarketID', 'DestCityMarketID',\n",
    "        'OriginAirportID', 'DestAirportID'\n",
    "    ]\n",
    "\n",
    "    # 显式转换为字符串类型（避免 FutureWarning）\n",
    "    df[categorical_features] = df[categorical_features].apply(\n",
    "        lambda col: col.astype(str, errors='ignore')\n",
    "    )\n",
    "\n",
    "    # 自动识别数值列\n",
    "    numeric_features = df.drop(columns=categorical_features).select_dtypes(include=[np.number]).columns.tolist()\n",
    "\n",
    "    print(\"分类变量：\", categorical_features)\n",
    "    print(\"数值变量：\", numeric_features)\n",
    "\n",
    "    # 缺失值填充 + 标准化\n",
    "    numeric_transformer = Pipeline(steps=[\n",
    "        ('imputer', SimpleImputer(strategy='median')),\n",
    "        ('scaler', StandardScaler())])\n",
    "\n",
    "    # 缺失值填充 + One-Hot编码\n",
    "    categorical_transformer = Pipeline(steps=[\n",
    "        ('imputer', SimpleImputer(strategy='constant', fill_value='Unknown')),\n",
    "        ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "\n",
    "    preprocessor = ColumnTransformer(\n",
    "        transformers=[\n",
    "            ('num', numeric_transformer, numeric_features),\n",
    "            ('cat', categorical_transformer, categorical_features)])\n",
    "\n",
    "    processed_array = preprocessor.fit_transform(df)\n",
    "\n",
    "    # 如果是稀疏矩阵，转换为 numpy array\n",
    "    try:\n",
    "        processed_array = processed_array.toarray()\n",
    "    except AttributeError:\n",
    "        pass\n",
    "\n",
    "    # 获取 OneHotEncoder 的特征名\n",
    "    cat_pipeline = preprocessor.named_transformers_['cat']\n",
    "    onehot_step = cat_pipeline.named_steps['onehot']\n",
    "    feature_names_cat = onehot_step.get_feature_names_out().tolist() if hasattr(onehot_step, 'get_feature_names_out') else []\n",
    "\n",
    "    # 拼接特征名\n",
    "    feature_names = numeric_features + feature_names_cat\n",
    "\n",
    "    # 转换为 DataFrame\n",
    "    df_processed = pd.DataFrame(processed_array, columns=feature_names)\n",
    "    print(\" processed_array shape:\", df_processed.shape)\n",
    "\n",
    "    # 异常值检测\n",
    "    print(\"\\n 开始异常值检测...\")\n",
    "    detect_outliers(df[numeric_features], numeric_features)\n",
    "\n",
    "    # 数据质量报告（预处理后）\n",
    "    print(\"\\n 预处理后数据质量报告：\")\n",
    "    missing_summary = df_processed.isnull().sum()\n",
    "    missing_rate = (missing_summary / df_processed.shape[0]) * 100\n",
    "    quality_report = pd.DataFrame({\n",
    "        'Missing Count': missing_summary,\n",
    "        'Missing Rate (%)': missing_rate,\n",
    "        'Unique Count': df_processed.nunique(),\n",
    "        'Data Type': df_processed.dtypes\n",
    "    })\n",
    "    print(quality_report)\n",
    "\n",
    "    print(\"【完成】数据预处理完成\")\n",
    "    return df_processed, preprocessor\n",
    "raw_df = pd.read_csv(\"data/processed/MarketFarePredictionData.csv\")\n",
    "sample_df = raw_df.head(100_000).copy()\n",
    "\n",
    "print(\"正在进行数据预处理...\")\n",
    "cleaned_df, _ = preprocess_data(sample_df)\n",
    "\n",
    "output_path = \"data/preprocessed/cleaned_data.parquet\"\n",
    "os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
    "cleaned_df.to_parquet(output_path)\n",
    "print(f\" 已保存预处理后的数据到 {output_path}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5d190183-1b8d-4f9b-b29f-0c8861986656",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "【开始】特征构造...\n",
      "✅ 新增特征 fare_density\n",
      "✅ 新增离散化特征 fare_density_category\n",
      "🧮 正在进行 One-Hot 编码...\n",
      "【开始】特征选择...\n",
      "✅ 筛选出前50个重要特征\n",
      "\n",
      "📊 特征工程后数据质量报告：\n",
      "                              Missing Count  Missing Rate (%)  Unique Count  \\\n",
      "MktMilesFlown                             0               0.0            87   \n",
      "fare_density                              0               0.0           914   \n",
      "NonStopMiles                              0               0.0           244   \n",
      "x0_6                                      0               0.0             2   \n",
      "Carrier_freq                              0               0.0            23   \n",
      "x0_20                                     0               0.0             2   \n",
      "CarrierPax                                0               0.0           264   \n",
      "x3_1                                      0               0.0             2   \n",
      "Market_share                              0               0.0           261   \n",
      "x0_13                                     0               0.0             2   \n",
      "DestCityMarketID_freq                     0               0.0            43   \n",
      "Market_HHI                                0               0.0            84   \n",
      "x9_121                                    0               0.0             2   \n",
      "OriginCityMarketID_freq                   0               0.0            38   \n",
      "Pax                                       0               0.0            87   \n",
      "x2_2993                                   0               0.0             2   \n",
      "x7_134                                    0               0.0             2   \n",
      "ODPairID_freq                             0               0.0            87   \n",
      "MktCoupons                                0               0.0             3   \n",
      "x8_108                                    0               0.0             2   \n",
      "x6_1.0                                    0               0.0             2   \n",
      "x3_0                                      0               0.0             2   \n",
      "fare_density_category_Medium              0               0.0             2   \n",
      "x2_3003                                   0               0.0             2   \n",
      "x6_0.0                                    0               0.0             2   \n",
      "x10_188                                   0               0.0             2   \n",
      "fare_density_category_Low                 0               0.0             2   \n",
      "DestAirportID_freq                        0               0.0            61   \n",
      "Circuity                                  0               0.0          1638   \n",
      "x10_144                                   0               0.0             2   \n",
      "x2_3510                                   0               0.0             2   \n",
      "x0_10                                     0               0.0             2   \n",
      "OriginAirportID_freq                      0               0.0            56   \n",
      "fare_density_category_High                0               0.0             2   \n",
      "x8_196                                    0               0.0             2   \n",
      "x8_159                                    0               0.0             2   \n",
      "x9_270                                    0               0.0             2   \n",
      "x8_110                                    0               0.0             2   \n",
      "x10_189                                   0               0.0             2   \n",
      "x7_158                                    0               0.0             2   \n",
      "x0_19                                     0               0.0             2   \n",
      "x2_1995                                   0               0.0             2   \n",
      "x4_1                                      0               0.0             2   \n",
      "x4_0                                      0               0.0             2   \n",
      "x10_242                                   0               0.0             2   \n",
      "x10_263                                   0               0.0             2   \n",
      "x7_64                                     0               0.0             2   \n",
      "x2_2292                                   0               0.0             2   \n",
      "x2_2258                                   0               0.0             2   \n",
      "x7_91                                     0               0.0             2   \n",
      "Average_Fare                              0               0.0           413   \n",
      "\n",
      "                             Data Type  \n",
      "MktMilesFlown                  float64  \n",
      "fare_density                   float64  \n",
      "NonStopMiles                   float64  \n",
      "x0_6                           float64  \n",
      "Carrier_freq                   float64  \n",
      "x0_20                          float64  \n",
      "CarrierPax                     float64  \n",
      "x3_1                           float64  \n",
      "Market_share                   float64  \n",
      "x0_13                          float64  \n",
      "DestCityMarketID_freq          float64  \n",
      "Market_HHI                     float64  \n",
      "x9_121                         float64  \n",
      "OriginCityMarketID_freq        float64  \n",
      "Pax                            float64  \n",
      "x2_2993                        float64  \n",
      "x7_134                         float64  \n",
      "ODPairID_freq                  float64  \n",
      "MktCoupons                     float64  \n",
      "x8_108                         float64  \n",
      "x6_1.0                         float64  \n",
      "x3_0                           float64  \n",
      "fare_density_category_Medium      bool  \n",
      "x2_3003                        float64  \n",
      "x6_0.0                         float64  \n",
      "x10_188                        float64  \n",
      "fare_density_category_Low         bool  \n",
      "DestAirportID_freq             float64  \n",
      "Circuity                       float64  \n",
      "x10_144                        float64  \n",
      "x2_3510                        float64  \n",
      "x0_10                          float64  \n",
      "OriginAirportID_freq           float64  \n",
      "fare_density_category_High        bool  \n",
      "x8_196                         float64  \n",
      "x8_159                         float64  \n",
      "x9_270                         float64  \n",
      "x8_110                         float64  \n",
      "x10_189                        float64  \n",
      "x7_158                         float64  \n",
      "x0_19                          float64  \n",
      "x2_1995                        float64  \n",
      "x4_1                           float64  \n",
      "x4_0                           float64  \n",
      "x10_242                        float64  \n",
      "x10_263                        float64  \n",
      "x7_64                          float64  \n",
      "x2_2292                        float64  \n",
      "x2_2258                        float64  \n",
      "x7_91                          float64  \n",
      "Average_Fare                   float64  \n",
      "✅ 已保存特征工程结果到 data/features/feature_engineered_data.parquet\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "def create_new_features(df):\n",
    "    \"\"\"\n",
    "    构造新特征\n",
    "    :param df: 预处理后的DataFrame\n",
    "    :return: 添加新特征后的DataFrame\n",
    "    \"\"\"\n",
    "    print(\"【开始】特征构造...\")\n",
    "    df['fare_density'] = df['Average_Fare'] / df['NonStopMiles']\n",
    "    print(\"✅ 新增特征 fare_density\")\n",
    "\n",
    "    # ✅ 新增特征：票价密度分段（特征变换 - 离散化）\n",
    "    bins = [0, 0.5, 1.0, 2.0, float('inf')]\n",
    "    labels = ['Very Low', 'Low', 'Medium', 'High']\n",
    "    df['fare_density_category'] = pd.cut(df['fare_density'], bins=bins, labels=labels)\n",
    "\n",
    "    print(\"✅ 新增离散化特征 fare_density_category\")\n",
    "    return df\n",
    "\n",
    "\n",
    "def select_important_features(X, y):\n",
    "    \"\"\"\n",
    "    使用随机森林评估特征重要性并筛选前50个特征\n",
    "    :param X: 特征矩阵\n",
    "    :param y: 目标变量\n",
    "    :return: 重要特征列表\n",
    "    \"\"\"\n",
    "    print(\"【开始】特征选择...\")\n",
    "    model = RandomForestRegressor(n_estimators=100, random_state=42)\n",
    "    model.fit(X, y)\n",
    "\n",
    "    importance = pd.Series(model.feature_importances_, index=X.columns).sort_values(ascending=False)\n",
    "    top_features = importance.head(50).index.tolist()\n",
    "    print(\"✅ 筛选出前50个重要特征\")\n",
    "\n",
    "    # ✅ 可视化特征重要性\n",
    "    os.makedirs(\"../data/output\", exist_ok=True)\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    importance.sort_values().plot(kind='barh')\n",
    "    plt.title(\"Top 50 Feature Importances\")\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(\"../data/output/feature_importance.png\")\n",
    "    plt.close()\n",
    "\n",
    "    return top_features\n",
    "\n",
    "\n",
    "def generate_quality_report(df):\n",
    "    \"\"\"\n",
    "    输出数据质量报告\n",
    "    :param df: DataFrame\n",
    "    \"\"\"\n",
    "    print(\"\\n📊 特征工程后数据质量报告：\")\n",
    "    missing_summary = df.isnull().sum()\n",
    "    missing_rate = (missing_summary / df.shape[0]) * 100\n",
    "    quality_report = pd.DataFrame({\n",
    "        'Missing Count': missing_summary,\n",
    "        'Missing Rate (%)': missing_rate,\n",
    "        'Unique Count': df.nunique(),\n",
    "        'Data Type': df.dtypes\n",
    "    })\n",
    "    print(quality_report)\n",
    "\n",
    "\n",
    "\n",
    "df = pd.read_parquet(\"data/preprocessed/cleaned_data.parquet\")\n",
    "df_with_new_features = create_new_features(df.copy())\n",
    "# 新增 One-Hot 编码\n",
    "print(\"🧮 正在进行 One-Hot 编码...\")\n",
    "df_encoded = pd.get_dummies(df_with_new_features, columns=['fare_density_category'], drop_first=True)\n",
    "# 假设目标变量是 'Average_Fare'\n",
    "X = df_encoded.drop(columns=['Average_Fare'])\n",
    "y = df_encoded['Average_Fare']\n",
    "selected_features = select_important_features(X, y)\n",
    "final_df = df_encoded[selected_features + ['Average_Fare']]\n",
    "generate_quality_report(final_df)\n",
    "output_path = \"data/features/feature_engineered_data.parquet\"\n",
    "os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
    "final_df.to_parquet(output_path)\n",
    "print(f\"已保存特征工程结果到 {output_path}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c2e49bce-cafc-4f4d-8814-9e2ef1eee935",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据形状：X=(3742, 50), y=(3742,)\n",
      "开始超参数搜索...\n",
      "Fitting 5 folds for each of 216 candidates, totalling 1080 fits\n",
      " 最佳参数组合： {'max_depth': 20, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50}\n",
      " 最佳得分（负均方误差）： 0.00997653589155525\n",
      " 测试集 MSE： 0.0038891949829626222\n",
      " 测试集 R² Score： 0.9962834470275647\n",
      "已保存最佳模型到 data/models/best_random_forest.pkl\n",
      " 已保存预测结果到 data/output/predictions.parquet\n"
     ]
    }
   ],
   "source": [
    "# hyperparameter_tuning.py\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import joblib\n",
    "# 加载数据\n",
    "df = pd.read_parquet(\"data/features/feature_engineered_data.parquet\")\n",
    "X = df.drop(columns=['Average_Fare'])\n",
    "y = df['Average_Fare']\n",
    "print(f\"数据形状：X={X.shape}, y={y.shape}\")\n",
    "# 划分训练集和测试集\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# 创建模型\n",
    "model = RandomForestRegressor(random_state=42)\n",
    "# 设置更丰富的超参数搜索空间\n",
    "param_grid = {\n",
    "    'n_estimators': [50, 100, 200],\n",
    "    'max_depth': [None, 10, 20, 30],\n",
    "    'min_samples_split': [2, 4, 6],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'max_features': ['sqrt', 'log2']\n",
    "}\n",
    "# 初始化 GridSearchCV\n",
    "grid_search = GridSearchCV(\n",
    "    estimator=model,\n",
    "    param_grid=param_grid,\n",
    "    cv=5,\n",
    "    scoring='neg_mean_squared_error',\n",
    "    n_jobs=-1,\n",
    "    verbose=1\n",
    ")\n",
    "\n",
    "# 开始超参数搜索\n",
    "print(\"开始超参数搜索...\")\n",
    "grid_result = grid_search.fit(X_train, y_train)\n",
    "# 获取最佳模型\n",
    "best_model = grid_result.best_estimator_\n",
    "y_pred = best_model.predict(X_test)\n",
    "mse = mean_squared_error(y_test, y_pred)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "# 输出结果\n",
    "print(\" 最佳参数组合：\", grid_result.best_params_)\n",
    "print(\" 最佳得分（负均方误差）：\", -grid_result.best_score_)\n",
    "print(\" 测试集 MSE：\", mse)\n",
    "print(\" 测试集 R² Score：\", r2)\n",
    "# 保存最佳模型\n",
    "os.makedirs(\"data/models\", exist_ok=True)\n",
    "model_path = \"data/models/best_random_forest.pkl\"\n",
    "joblib.dump(best_model, model_path)\n",
    "print(f\"已保存最佳模型到 {model_path}\")\n",
    "# 保存预测结果\n",
    "result_df = pd.DataFrame({\n",
    "    \"True_Value\": y_test,\n",
    "    \"Predicted_Value\": y_pred\n",
    "})\n",
    "output_path = \"data/output/predictions.parquet\"\n",
    "os.makedirs(os.path.dirname(output_path), exist_ok=True)\n",
    "result_df.to_parquet(output_path)\n",
    "print(f\" 已保存预测结果到 {output_path}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "148a9bba-e64c-49f2-985d-af3fc88c416f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 正在加载数据...\n",
      " 数据形状：X=(3742, 50), y=(3742,)\n",
      "\n",
      " 开始第 1 折交叉验证 \n",
      "第 1 折验证完成 - Val MSE: 0.002224, MAE: 0.012123, R²: 0.997875\n",
      "\n",
      " 开始第 2 折交叉验证 \n",
      "第 2 折验证完成 - Val MSE: 0.005288, MAE: 0.014503, R²: 0.994682\n",
      "\n",
      " 开始第 3 折交叉验证 \n",
      "第 3 折验证完成 - Val MSE: 0.001555, MAE: 0.010658, R²: 0.998444\n",
      "\n",
      " 开始第 4 折交叉验证 \n",
      "第 4 折验证完成 - Val MSE: 0.001636, MAE: 0.010757, R²: 0.998132\n",
      "\n",
      " 开始第 5 折交叉验证 \n",
      "第 5 折验证完成 - Val MSE: 0.001588, MAE: 0.009362, R²: 0.998533\n",
      "\n",
      " 五折平均评估指标：\n",
      " 平均 MSE: 0.002458\n",
      " 平均 MAE: 0.011481\n",
      " 平均 R² Score: 0.997533\n",
      " 已保存 KFold 最佳模型到 data/models/random_forest_kfold.pkl\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import joblib\n",
    "\n",
    "\n",
    "def plot_history(val_scores, fold_count):\n",
    "    \"\"\"\n",
    "    绘制每一折的 MSE 指标（英文标注）\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    plt.plot(val_scores, label='Validation MSE', marker='o')\n",
    "    plt.title('K-Fold Validation MSE Scores')\n",
    "    plt.xlabel('Fold')\n",
    "    plt.ylabel('MSE')\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "\n",
    "    # 创建输出目录并保存图像\n",
    "    output_dir = \"data/output/\"\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    plt.savefig(os.path.join(output_dir, \"rf_training_history.png\"))\n",
    "    plt.close()  # 非交互环境避免弹窗显示图像\n",
    "\n",
    "\n",
    "def train_model(X, y, n_splits=5, random_state=42):\n",
    "    \"\"\"\n",
    "    使用 KFold 交叉验证训练模型，并返回每折的评估结果和模型\n",
    "    \"\"\"\n",
    "    kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n",
    "    fold_no = 1\n",
    "    val_scores = []\n",
    "    mae_scores = []\n",
    "    r2_scores = []\n",
    "\n",
    "    for train_index, val_index in kf.split(X):\n",
    "        print(f\"\\n 开始第 {fold_no} 折交叉验证 \")\n",
    "        X_train, X_val = X[train_index], X[val_index]\n",
    "        y_train, y_val = y[train_index], y[val_index]\n",
    "\n",
    "        # 构建模型\n",
    "        model = RandomForestRegressor(\n",
    "            n_estimators=100,\n",
    "            max_depth=None,\n",
    "            min_samples_split=2,\n",
    "            min_samples_leaf=1,\n",
    "            n_jobs=-1,\n",
    "            random_state=random_state\n",
    "        )\n",
    "\n",
    "        # 训练模型\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        # 预测与评估\n",
    "        y_pred = model.predict(X_val)\n",
    "        mse = mean_squared_error(y_val, y_pred)\n",
    "        mae = mean_absolute_error(y_val, y_pred)\n",
    "        r2 = r2_score(y_val, y_pred)\n",
    "\n",
    "        val_scores.append(mse)\n",
    "        mae_scores.append(mae)\n",
    "        r2_scores.append(r2)\n",
    "\n",
    "        print(f\"第 {fold_no} 折验证完成 - Val MSE: {mse:.6f}, MAE: {mae:.6f}, R²: {r2:.6f}\")\n",
    "\n",
    "        # 保存每折的预测结果（可选）\n",
    "        result_df = pd.DataFrame({\n",
    "            \"Fold\": [fold_no] * len(y_val),\n",
    "            \"True_Value\": y_val,\n",
    "            \"Predicted_Value\": y_pred\n",
    "        })\n",
    "        result_path = \"data/output/kfold_predictions.parquet\"\n",
    "        if fold_no == 1:\n",
    "            result_df.to_parquet(result_path, engine='pyarrow', index=False)\n",
    "        else:\n",
    "            prev_df = pd.read_parquet(result_path)\n",
    "            combined_df = pd.concat([prev_df, result_df], ignore_index=True)\n",
    "            combined_df.to_parquet(result_path, engine='pyarrow', index=False)\n",
    "\n",
    "        fold_no += 1\n",
    "\n",
    "    # 输出平均指标\n",
    "    avg_mse = np.mean(val_scores)\n",
    "    avg_mae = np.mean(mae_scores)\n",
    "    avg_r2 = np.mean(r2_scores)\n",
    "\n",
    "    print(\"\\n 五折平均评估指标：\")\n",
    "    print(f\" 平均 MSE: {avg_mse:.6f}\")\n",
    "    print(f\" 平均 MAE: {avg_mae:.6f}\")\n",
    "    print(f\" 平均 R² Score: {avg_r2:.6f}\")\n",
    "\n",
    "    return model, avg_mse, avg_mae, avg_r2\n",
    "\n",
    "\n",
    "\n",
    "print(\" 正在加载数据...\")\n",
    "df = pd.read_parquet(\"data/features/feature_engineered_data.parquet\")\n",
    "X = df.drop(columns=['Average_Fare']).values\n",
    "y = df['Average_Fare'].values\n",
    "print(f\" 数据形状：X={X.shape}, y={y.shape}\")\n",
    "# 训练模型\n",
    "best_model, avg_mse, avg_mae, avg_r2 = train_model(X, y)\n",
    "# 保存模型\n",
    "os.makedirs(\"data/models\", exist_ok=True)\n",
    "model_path = \"data/models/random_forest_kfold.pkl\"\n",
    "joblib.dump(best_model, model_path)\n",
    "print(f\" 已保存 KFold 最佳模型到 {model_path}\")\n",
    "# 可视化 MSE 曲线\n",
    "plot_history(avg_mse, fold_count=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d3e2fdee-ec55-4453-bbc9-af5114e615cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在加载数据...\n",
      " 正在绘制特征分布图...\n",
      " 正在绘制特征相关性热力图...\n",
      " 正在绘制真实值 vs 预测值对比图...\n",
      " 正在复制特征重要性图...\n",
      "特征重要性图不存在，请先运行特征工程模块\n",
      " 正在复制模型训练曲线...\n",
      "MSE 曲线图不存在，请先运行模型训练模块\n",
      " 所有可视化图表已保存至：../data/output/visualizations\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "def set_plot_style():\n",
    "    \"\"\"设置全局绘图样式\"\"\"\n",
    "    sns.set(style=\"whitegrid\", font_scale=1.2)  # 使用 seaborn 全局样式\n",
    "\n",
    "\n",
    "\n",
    "def plot_feature_distribution(df, output_dir):\n",
    "    \"\"\"绘制数值特征的分布直方图\"\"\"\n",
    "    numeric_features = df.select_dtypes(include=[np.number]).columns\n",
    "    fig, axes = plt.subplots(4, 4, figsize=(20, 15))\n",
    "    axes = axes.flatten()\n",
    "\n",
    "    for i, col in enumerate(numeric_features[:16]):  # 显示前16个数值特征\n",
    "        sns.histplot(df[col], ax=axes[i], kde=True)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(os.path.join(output_dir, \"feature_distributions.png\"))\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_correlation_matrix(df, output_dir):\n",
    "    \"\"\"绘制特征相关性热力图\"\"\"\n",
    "    numeric_df = df.select_dtypes(include=[np.number])\n",
    "    corr = numeric_df.corr()\n",
    "    plt.figure(figsize=(12, 10))\n",
    "    sns.heatmap(corr, annot=False, cmap='coolwarm', fmt=\".2f\", linewidths=.5)\n",
    "    plt.title(\"Feature Correlation Matrix\")\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(os.path.join(output_dir, \"correlation_matrix.png\"))\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_true_vs_predicted(output_dir):\n",
    "    \"\"\"绘制测试集中真实值 vs 预测值对比图\"\"\"\n",
    "    pred_df = pd.read_parquet(\"../data/output/predictions.parquet\")\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    sns.scatterplot(x=\"True_Value\", y=\"Predicted_Value\", data=pred_df, alpha=0.6)\n",
    "    plt.plot([pred_df[\"True_Value\"].min(), pred_df[\"True_Value\"].max()],\n",
    "             [pred_df[\"True_Value\"].min(), pred_df[\"True_Value\"].max()],\n",
    "             'r--')  # 添加理想线\n",
    "    plt.title(\"True vs Predicted Values (Test Set)\")\n",
    "    plt.xlabel(\"True Value\")\n",
    "    plt.ylabel(\"Predicted Value\")\n",
    "    plt.grid(True)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(os.path.join(output_dir, \"true_vs_predicted.png\"))\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "def plot_feature_importance(output_dir):\n",
    "    \"\"\"加载并显示特征重要性图\"\"\"\n",
    "    importance_path = \"data/output/feature_importance.png\"\n",
    "    if os.path.exists(importance_path):\n",
    "        plt.figure(figsize=(12, 8))\n",
    "        img = plt.imread(importance_path)\n",
    "        plt.imshow(img)\n",
    "        plt.axis('off')\n",
    "        plt.title(\"Top Feature Importances\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(os.path.join(output_dir, \"feature_importance.png\"))\n",
    "        plt.close()\n",
    "    else:\n",
    "        print(\"特征重要性图不存在，请先运行特征工程模块\")\n",
    "\n",
    "\n",
    "def plot_model_mse_curve(output_dir):\n",
    "    \"\"\"加载并显示训练过程中的 MSE 曲线\"\"\"\n",
    "    mse_path = \"data/output/rf_training_history.png\"\n",
    "    if os.path.exists(mse_path):\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        img = plt.imread(mse_path)\n",
    "        plt.imshow(img)\n",
    "        plt.axis('off')\n",
    "        plt.title(\"Validation MSE Across Folds\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(os.path.join(output_dir, \"model_mse_curve.png\"))\n",
    "        plt.close()\n",
    "    else:\n",
    "        print(\"MSE 曲线图不存在，请先运行模型训练模块\")\n",
    "\n",
    "\n",
    "set_plot_style()\n",
    "viz_dir = \"../data/output/visualizations\"\n",
    "os.makedirs(viz_dir, exist_ok=True)\n",
    "print(\"正在加载数据...\")\n",
    "raw_df = pd.read_parquet(\"data/preprocessed/cleaned_data.parquet\")\n",
    "engineered_df = pd.read_parquet(\"data/features/feature_engineered_data.parquet\")\n",
    "print(\" 正在绘制特征分布图...\")\n",
    "plot_feature_distribution(raw_df, viz_dir)\n",
    "print(\" 正在绘制特征相关性热力图...\")\n",
    "plot_correlation_matrix(raw_df, viz_dir)\n",
    "print(\" 正在绘制真实值 vs 预测值对比图...\")\n",
    "plot_true_vs_predicted(viz_dir)\n",
    "print(\" 正在复制特征重要性图...\")\n",
    "plot_feature_importance(viz_dir)\n",
    "print(\" 正在复制模型训练曲线...\")\n",
    "plot_model_mse_curve(viz_dir)\n",
    "print(f\" 所有可视化图表已保存至：{viz_dir}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8efd5fda-92d9-4bb2-ab64-ef068cf4beff",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
